\begin{table}[t!]
\scriptsize
\caption[]{\footnotesize{}Losses obtained for all optimizers and benchmarks . We report means and standard deviation across 10 runs of each optimizer. For each benchmark, bold face indicates the best mean loss, and underlined values are not statistically significantly different from the best according to an unpaired t-test (with p=0.05). For \hpnnet{} we also provide results for half the function evaluation budget to quantify the improvement over time.
%The GPU experiments did not finish due to an unscheduled 2-week outage of the cluster we used; for them, we used (given as a superscript number); we will update these rows for the final version.
%
%The absolute values with and without cross-validation are incomparable as they are based on different data set splits.
\label{tab:results}}
\vspace*{-0.1cm}
\begin{center}
\begin{tabularx}{\textwidth}{lr|Xr|Xr|Xr}
\toprule
\multicolumn{2}{l}{} &\multicolumn{2}{c}{\bf SMAC} &\multicolumn{2}{c}{\bf Spearmint} &\multicolumn{2}{c}{\bf TPE} \\
\multicolumn{1}{l}{\bf Experiment} &\multicolumn{1}{r}{\#evals}
&\multicolumn{1}{l}{Valid.\ loss} &\multicolumn{1}{r}{Best loss}
&\multicolumn{1}{l}{Valid.\ loss} &\multicolumn{1}{r}{Best loss}
&\multicolumn{1}{l}{Valid.\ loss} &\multicolumn{1}{r}{Best loss}
\\ 
\toprule
%exp | #eval |smac {v|num}| spear {v|num} | tpe {v|num}
branin~(0.398)     & 200 & 0.655$\pm$0.27 & 0.408 & 
                           \underline{\textbf{0.398}}$\pm$0.00  & \textbf{0.398} &
                           0.526$\pm$ 0.13  & 0.422 \\                           
har6~(-3.322)    & 200 & \underline{-2.977}$\pm$0.11 & -3.154 & 
                           \underline{\textbf{-3.133}}$\pm$0.41 & \textbf{-3.322}	 & 
                           \underline{-2.823}$\pm$0.18 & -3.039 \\
\midrule
Log.Regression     & 100 & 8.6$\pm$0.9 & 7.7 & 
                           \underline{\textbf{7.3}}$\pm$0.2 & \textbf{7.0} & 
                           8.2$\pm$0.6 & 7.5 \\
LDA ongrid         &  50 & \underline{\textbf{1269.6}}$\pm$2.9  & \textbf{1266.2} & 
                           \underline{1272.6}$\pm$10.3 & \textbf{1266.2} & 
                           \underline{1271.5}$\pm$3.5 & \textbf{1266.2} \\
SVM ongrid         & 100 & \underline{\textbf{24.1}}$\pm$0.1 & \textbf{24.1} & 
                           \underline{24.6}$\pm$0.9 & \textbf{24.1} & 
                           \underline{24.2}$\pm$0.0 & \textbf{24.1} \\
\midrule
%\hpnnet{} convex        & 50 & \underline{20.9}$\pm$1.6 & \textbf{17.47} & 
%                           \underline{22.1}$\pm$3.2 & 20.33 & 
%                           \underline{\textbf{20.8}}$\pm$1.3 & 18.33  \\
\hpnnet{} convex        & 100 & \underline{\textbf{19.5}}$\pm$1.5 & \textbf{17.0} & 
                           20.6$\pm$0.3 & 20.1 & 
                           \underline{\textbf{19.5}}$\pm$1.6 & 17.4  \\
\hpnnet{} convex        & 200 & \underline{\textbf{18.3}}$\pm$1.9 & \textbf{15.2} & 
                           20.0$\pm$0.9 & 17.3 & 
                           \underline{18.5}$\pm$1.4 & 16.2\smallskip\\
%
%\hpnnet{} MRBI          & 50 &  \underline{54.3}$\pm$2.3 & 49.95 & 
%                                 \underline{55.6}$\pm$4.9 & 47.80 & 
%                                 \underline{\textbf{52.4}}$\pm$2.0 & \textbf{49.50} \\                          		        
\hpnnet{} MRBI          & 100 &  \underline{51.5}$\pm$2.8 & \textbf{46.1} & 
                                 \underline{52.2}$\pm$3.3 & 46.5 & 
                                 \underline{\textbf{50.0}}$\pm$1.7 & 47.3 \\
\hpnnet{} MRBI          & 200 & \underline{\textbf{48.3}}$\pm$1.80 & \textbf{46.1} & 
                                                    51.4$\pm$3.2 & 46.5 & 
                                        \underline{48.9}$\pm$1.4 & 46.9\smallskip\\
%																			
\hpdbnet{} convex       & 100 & \underline{\textbf{16.4}}$\pm$1.2 & \textbf{14.5} &
                                \underline{20.74}$\pm$6.9 & 15.5 & 
                       \underline{17.29}$\pm$1.7 & 15.3 \\
\hpdbnet{} convex       & 200 & \underline{\textbf{15.4}}$\pm$0.8 & \textbf{14.0} &
                                \underline{17.45}$\pm$5.6 & 14.6 & 
                                16.1$\pm$0.5 & 15.3 \\
%\hpdbnet{} convex & 200 & \underline{0.159}$^{(1)}$ & { } &
%                          \underline{0.138}$^{(1)}$ & { } &
%                          \underline{\textbf{0.135}}$^{(1)}$ & { } % the 1 run on meta-gpu on this before submission => SMAC looked worst\\

%\hpdbnet{} convex & 100 & \underline{17.8}$\pm$4.2$^{(5)}$ & 14.1 &
%                          \underline{24.7}$\pm$7.9$^{(5)}$ & 16.6 &
%                          \underline{\textbf{16.8}}$\pm$1.8$^{(5)}$ & 14.9 % 5 runs on meta-gpu after submission => SMAC is actually better after all, both on the Parallel cluster (see above) and also on this archictecture\\
% Branin, CUT 200
%      SMAC:    0.655 (min:    0.408, std: 0.266)
%     spear:    0.398 (min:    0.398, std: 0.000)
%       TPE:    0.526 (min:    0.422, std: 0.124)

% HAR6, CUT 200
%      SMAC:   -2.977 (min:   -3.154, std: 0.108)
% spearmint:   -3.133 (min:   -3.322, std: 0.409)
%       TPE:   -2.823 (min:   -3.039, std: 0.177)
                      
% MRBI NOCV, CUT = 200
%      SMAC:    48.28 (min:    46.10, std: 1.8)
%     spear:    51.35 (min:    46.45, std: 3.2)                
%       TPE:    48.93 (min:    46.90, std: 1.4)
% MRBI NOCV, CUT = 100
%      SMAC:    51.51 (min:    46.10, std: 2.8)
%     spear:    52.23 (min:    46.45, std: 3.3)
%       TPE:    50.02 (min:    47.25, std: 1.7)

\midrule
Auto-WEKA%\footnotemark
& 30h & \underline{\textbf{27.5}}$\pm$4.9 & \textbf{22.3} &
 	40.64$\pm$7.2 & 31.9 &
35.5$\pm$2.9 & 28.8\\
\midrule
Log.Regression 5CV & 500 folds  & \underline{\textbf{8.1}}$\pm$0.2 & \textbf{7.8} & 
                           \underline{8.2}$\pm$0.1 & 7.9 &
                           8.9$\pm$0.5 & 8.1 \\
\hpnnet{} convex 5CV    & 500 folds & \underline{\textbf{18.2}}$\pm$1.5 & \textbf{16.9} & 
                          		        23.0$\pm$5.0 & 19.7 & 
                          		        20.9$\pm$1.3 & 18.6 \\
\hpnnet{} MRBI 5CV      & 500 folds  & \underline{\textbf{47.9}}$\pm$0.7 &         47.2  & 
                                                   52.8$\pm$5.1  & \textbf{46.6} & 
                                                           50.8 $\pm$1.4 &         48.2  \\
          
% MRBI CV, 500 folds
%      SMAC:    47.9 (min:    47.23, std: 0.7)
%     spear:    52.8 (min:    46.56, std: 5.1)
%       TPE:    50.8 (min:    48.16, std: 1.4)

%DBNet MRBI & 200 & & & & & & \\
\bottomrule
\vspace*{-0.3cm}
\end{tabularx}
\end{center}


%
%\begin{minipage}{\textwidth}
%		\begin{minipage}{0.33\textwidth}	
%        \begin{figure}[H]
%                \includegraphics[width=\linewidth,height=2.5cm]{graphics/LDA_Trajectory_alternative.pdf}
%								\vspace*{-0.2cm}
%                \caption{\footnotesize{}Response values obtained \& incumbent over time for one run of each optimizer on \textsc{LDA ongrid}.\label{fig:lda_trajectory}}
%        \end{figure}%
%		\end{minipage}		
%		\quad
%		\begin{minipage}{0.305\textwidth}	
%        \begin{figure}[H]
%                \includegraphics[width=\textwidth]{graphics/NNet_convex_nocv_ErrorTrace.pdf}
%								\vspace*{-0.2cm}
%               \caption{\footnotesize{}Mean $\pm$ stddev of best loss over time, \hpnnet{} convex\label{fig:nnet_convex_nocv_err}}
%        \end{figure}%
%		\end{minipage}		
%		\quad
%		\begin{minipage}{0.305\textwidth}	
%        \begin{figure}[H]
%               \includegraphics[width=\textwidth]{graphics/NNet_mrbi_nocv_ErrorTrace.pdf}
%								\vspace*{-0.2cm}
%               \caption{\footnotesize{}Mean $\pm$ stddev of best loss over time, \hpnnet{} MRBI\label{fig:nnet_mrbi_nocv_err}}
%        \end{figure}%
%		\end{minipage}		
%\end{minipage}

\end{table}
